{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#测试样本的数量\n",
    "n_sample=100\n",
    "def gen_label_pred(n_sample):\n",
    "    \"\"\"\n",
    "    随机生成n个样本的标签和预测值\n",
    "    \"\"\"\n",
    "    label1 = [0 for _ in range(n_sample//2)]\n",
    "    pred1 = [random.random()*0.7 for _ in range(n_sample//2)]\n",
    "    label2 = [1 for _ in range(n_sample//2)]\n",
    "    pred2 = [(random.random() + 1)/2 for _ in range(n_sample//2)]\n",
    "    labels = label1 + label2\n",
    "    preds = pred1 + pred2\n",
    "    return labels,preds\n",
    "labels,preds = gen_label_pred(n_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probability</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.178497</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.037655</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.376118</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.396839</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.563260</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   probability  label\n",
       "0     0.178497      0\n",
       "1     0.037655      0\n",
       "2     0.376118      0\n",
       "3     0.396839      0\n",
       "4     0.563260      0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.DataFrame(index=range(0,n_sample),columns=('probability','label'))\n",
    "data['label']=np.array(labels)\n",
    "data['probability']=np.array(preds)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Zhao\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \n",
      "C:\\Users\\Zhao\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "C:\\Users\\Zhao\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:4: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  after removing the cwd from sys.path.\n",
      "C:\\Users\\Zhao\\Miniconda3\\lib\\site-packages\\ipykernel_launcher.py:5: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "cm=np.arange(4).reshape(2,2)\n",
    "cm[0,0]=len(data[data['label']==0][data['probability']<0.5]) #TN\n",
    "cm[0,1]=len(data[data['label']==0][data['probability']>=0.5])#FP\n",
    "cm[1,0]=len(data[data['label']==1][data['probability']<0.5]) #FN\n",
    "cm[1,1]=len(data[data['label']==1][data['probability']>=0.5])#TP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[36, 14],\n",
       "       [ 0, 50]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAEYCAYAAACgIGhkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE+hJREFUeJzt3Xuc1XWdx/HXe2YYAUEuoiggAiIQ\n+lADMR90c1fXdEWli6WSq6sraWlbZq2lu2uam+Z29VJpmZWlUq2btyJrH+b9ipqRgCgSNwVBUUGG\nmeGzf5zf0BHne+YMzG9+B3g/H4/z4Pwu5/d9H2De8/v9zu+co4jAzKw9dUUHMLPa5YIwsyQXhJkl\nuSDMLMkFYWZJLggzS3JBbIMk9ZJ0m6TVkn6xBduZJul3XZmtKJLeK2lu0Tm2NvJ1EMWRdCJwDjAO\neB14ErgkIu7bwu2eBJwNTI6Ili0OWuMkBbB3RMwvOsu2xnsQBZF0DvAt4L+AwcBw4Grg2C7Y/J7A\nvO2hHKohqaHoDFutiPCtm29AP+AN4LgK6+xAqUCWZrdvATtkyw4BFgOfA5YDy4B/zpZ9GVgPNGdj\nnAZcCNxQtu0RQAAN2fQpwPOU9mIWANPK5t9X9rjJwKPA6uzPyWXL7gYuBu7PtvM7YFDiubXl/0JZ\n/qnAPwLzgFXAl8rWPwh4EHg1W/dKoDFbdk/2XNZkz/djZdv/N+BF4Kdt87LH7JWNMSGbHgK8DBxS\n9P+NWrsVHmB7vAFHAC1tP6CJdS4CHgJ2BXYBHgAuzpYdkj3+IqBH9oO1FhiQLd+0EJIFAewIvAaM\nzZbtDuyT3d9YEMBA4BXgpOxxJ2TTO2fL7waeA8YAvbLpSxPPrS3/f2T5TwdWAD8H+gL7AOuAUdn6\nE4GDs3FHAM8AnynbXgCj29n+ZZSKtld5QWTrnJ5tpzcwE/jvov9f1OLNhxjF2Bl4OSofAkwDLoqI\n5RGxgtKewUlly5uz5c0RcSel355jNzPPBmBfSb0iYllEzG5nnaOAZyPipxHREhE3AnOAo8vW+VFE\nzIuIN4EZwAEVxmymdL6lGbgJGAR8OyJez8afDewHEBGPR8RD2bgvAN8H3l/Fc/rPiGjK8rxFRFwL\nPAs8TKkUz+9ge9slF0QxVgKDOjg2HgIsLJtemM3buI1NCmYt0KezQSJiDaXd8jOAZZLukDSuijxt\nmYaWTb/YiTwrI6I1u9/2A/xS2fI32x4vaYyk2yW9KOk1SudtBlXYNsCKiFjXwTrXAvsCV0REUwfr\nbpdcEMV4kNIu9NQK6yyldLKxzfBs3uZYQ2lXus1u5QsjYmZE/AOl36RzKP3gdJSnLdOSzczUGd+l\nlGvviNgJ+BKgDh5T8eU5SX0ondf5IXChpIFdEXRb44IoQESspnT8fZWkqZJ6S+oh6UhJX8tWuxG4\nQNIukgZl69+wmUM+CbxP0nBJ/YAvti2QNFjSMZJ2BJooHaq0trONO4Exkk6U1CDpY8B44PbNzNQZ\nfSmdJ3kj27s5c5PlLwGjOrnNbwOPR8S/AHcA39vilNsgF0RBIuIblK6BuIDSCbpFwFnA/2arfAV4\nDPgT8DQwK5u3OWPdBdycbetx3vpDXUfp1ZCllM7svx/4ZDvbWAlMydZdSekViCkR8fLmZOqkc4ET\nKb06ci2l51LuQuDHkl6V9NGONibpWEonis/IZp0DTJA0rcsSbyN8oZSZJXkPwsySXBBmluSCMLMk\nF4SZJdXUm1gaeveLHv0GFx3DOmn4zr07XslqyotLFrH6lZUdXUtSWwXRo99gRpxyRdExrJOuPHli\n0RGskz75kcOqWs+HGGaW5IIwsyQXhJkluSDMLMkFYWZJLggzS3JBmFmSC8LMklwQZpbkgjCzJBeE\nmSW5IMwsyQVhZkkuCDNLckGYWZILwsySXBBmluSCMLMkF4SZJbkgzCzJBWFmSS4IM0tyQZhZkgvC\nzJJcEGaW5IIwsyQXhJkluSDMLMkFYWZJLggzS3JBmFmSC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVh\nZkkuCDNLckGYWZILwsySXBBmluSCMLMkF4SZJbkgzCzJBWFmSS4IM0tqKDrAtmZDy3r++rNziZZm\nIlrpO/a97PLek4gIXr7nx7w2515UV0f/dx7FwAOnFh3XMpef/2kevvsu+g8cxA9uu/cty2ZcdxXX\nXH4hv3pgDv0G7FxQwmK4ILqY6nsw/ITLqGvsRbS2sPCGz9Fn1IE0rVxE82srGDX9WqQ6Wta8WnRU\nK/OBqccz9cTTuOy8s94yf/myJTz+wN3suvuwYoIVzIcYXUwSdY29AIgNLcSGFpB49YnbGfSeaUil\nv/KGHfsXGdM2sd+kyfTtP+Bt87976QVMP/c/kVRAquJ5DyIHsaGVF64/m/WvLGXAhKPpNWQc619Z\nxmvP/JE35j1Afe9+DD7sTBoHDi06qlXwwP/9lkGDd2evcfsWHaUwue5BSDpC0lxJ8yWdl+dYtUR1\n9Yw89WpGf+oG1i2bS9OKF4jWZlTfyIhTrqD//kew7M5vFB3TKlj35lp+/v1vcvLZ281/23blVhCS\n6oGrgCOB8cAJksbnNV4tqu/Zh97D9+ON5x+jR99B9B37bgD6jHk3TSsWFJzOKlm66AVeXPxXPjH1\nEKYdOoEVLy3ljA8fyqoVLxUdrVvleYhxEDA/Ip4HkHQTcCzwlxzHLFzL2ldRXQP1PfuwobmJNS88\nwc4Hf5Q+YyazduFTNPbfjbV//RONA3x4UctGjRnPL+9/ZuP0tEMncPUv7/KrGF1oKLCobHox8K5N\nV5I0HZgO0LDTrjnG6R4tb6xi2e1fh2glIthp3PvoM/pd9Bq2D0tvu4xXHrsF9ejJbkd+tuioVuaS\nz03nqUfuZ/Wrqzj+kP04+awvcORHPl50rMLlWRDtnfaNt82IuAa4BqDX7mPetnxr03PXUYw89aq3\nza/v2Yc9jru4gERWjfO/fk3F5T/7w6xuSlJb8jxJuRjYo2x6GLA0x/HMrIvlWRCPAntLGimpETge\nuDXH8cysi+V2iBERLZLOAmYC9cB1ETE7r/HMrOvleqFURNwJ3JnnGGaWH19qbWZJLggzS3JBmFmS\nC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVhZkkuCDNLckGYWZILwsySXBBmluSCMLMkF4SZJbkgzCzJ\nBWFmSS4IM0tyQZhZkgvCzJJcEGaW5IIwsyQXhJkluSDMLMkFYWZJLggzS0p+9Z6knSo9MCJe6/o4\nZlZLKn0352wgAJXNa5sOYHiOucysBiQLIiL26M4gZlZ7qjoHIel4SV/K7g+TNDHfWGZWCzosCElX\nAn8HnJTNWgt8L89QZlYbKp2DaDM5IiZIegIgIlZJasw5l5nVgGoOMZol1VE6MYmknYENuaYys5pQ\nTUFcBfwK2EXSl4H7gMtyTWVmNaHDQ4yI+Imkx4HDslnHRcSf841lZrWgmnMQAPVAM6XDDF99abad\nqOZVjPOBG4EhwDDg55K+mHcwMyteNXsQHwcmRsRaAEmXAI8DX80zmJkVr5rDhYW8tUgagOfziWNm\ntaTSm7W+Semcw1pgtqSZ2fThlF7JMLNtXKVDjLZXKmYDd5TNfyi/OGZWSyq9WeuH3RnEzGpPhycp\nJe0FXAKMB3q2zY+IMTnmMrMaUM1JyuuBH1H6HIgjgRnATTlmMrMaUU1B9I6ImQAR8VxEXEDp3Z1m\nto2r5jqIJkkCnpN0BrAE2DXfWGZWC6opiM8CfYBPUzoX0Q84Nc9QZlYbqnmz1sPZ3df524fGmNl2\noNKFUreQfQZEeyLiQ7kkMrOaUWkP4spuS5EZP2Qn7r/4A909rG2hAZPOKjqCdVLT80urWq/ShVJ/\n6LI0ZrZV8mc7mFmSC8LMkqouCEk75BnEzGpPNZ8odZCkp4Fns+n9JV2RezIzK1w1exDfAaYAKwEi\n4il8qbXZdqGagqiLiIWbzGvNI4yZ1ZZqLrVeJOkgICTVA2cD8/KNZWa1oJo9iDOBc4DhwEvAwdk8\nM9vGVfNejOXA8d2QxcxqTDWfKHUt7bwnIyKm55LIzGpGNecgfl92vyfwQWBRPnHMrJZUc4hxc/m0\npJ8Cd+WWyMxqxuZcaj0S2LOrg5hZ7anmHMQr/O0cRB2wCjgvz1BmVhsqFkT2WZT7U/ocSoANEZH8\nEBkz27ZUPMTIyuCWiGjNbi4Hs+1INecgHpE0IfckZlZzKn0mZUNEtADvAU6X9BywhtIX6EREuDTM\ntnGVzkE8AkwApnZTFjOrMZUKQlD6Nq1uymJmNaZSQewi6ZzUwoj4Rg55zKyGVCqIekrfqKVuymJm\nNaZSQSyLiIu6LYmZ1ZxKL3N6z8FsO1epIA7tthRmVpOSBRERq7oziJnVHn9xjpkluSDMLMkFYWZJ\nLggzS3JBmFmSC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVhZkkuCDNLckGYWZILwsySXBBmluSCMLMk\nF4SZJbkgzCzJBWFmSS4IM0tyQeTsdzN/y377jGWfcaO5/GuXFh3HKlg3+yc0zbmRpjk30TR3BgDR\nso71839N019uYP38XxMt6wpO2b0qfXGObaHW1lY+8+lPccdv7mLosGG85+BJTJlyDO8YP77oaJbQ\nOHoqaui1cbpl+Szq+g6jYfBEWl56nJbls+gxZHKBCbuX9yBy9Ogjj7DXXqMZOWoUjY2NHPex47n9\ntl8XHcs6YcPqBdQPHAdA/cBxbFi9oOBE3csFkaOlS5cwbNgeG6eHDh3GkiVLCkxklUiw/rlbaZo7\ng5aXZwMQzWtRjx1Ly3vsSLS8WWTEbpfbIYak64ApwPKI2DevcWpZRLxtnuRvNKxVjXt/uFQCzWtZ\n/9yt1PUcUHSkwuW5B3E9cESO2695Q4cOY/HiRRunlyxZzJAhQwpMZJX8bU+hN3X9RrFh7UuoR2+i\neQ0A0bzmLecntge5FURE3ANs11/fd+CkScyf/ywvLFjA+vXr+cXNN3HUlGOKjmXtiNZmonX9xvsb\nXl+Eeg6kbqcRtK6aA0DrqjnU9RtZZMxuV/irGJKmA9MB9hg+vOA0XauhoYFvfvtKjj7qA7S2tnLy\nKacyfp99io5l7YiWtTQv+E02tYH6/mOo32lP6noPpvmF39K08hnU2IceI7avnWK1d5zcZRuXRgC3\nV3sOYuLEA+P+hx/LLY/lY8Cks4qOYJ3UNHcGG9Yu7/CEmF/FMLMkF4SZJeVWEJJuBB4ExkpaLOm0\nvMYys3zkdpIyIk7Ia9tm1j18iGFmSS4IM0tyQZhZkgvCzJJcEGaW5IIwsyQXhJkluSDMLMkFYWZJ\nLggzS3JBmFmSC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVhZkkuCDNLckGYWZILwsySXBBmluSCMLMk\nF4SZJbkgzCzJBWFmSS4IM0tyQZhZkgvCzJJcEGaW5IIwsyQXhJkluSDMLMkFYWZJLggzS3JBmFmS\nC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVhZkkuCDNLckGYWZILwsySXBBmluSCMLMkF4SZJbkgzCzJ\nBWFmSYqIojNsJGkFsLDoHDkYBLxcdAjrtG35323PiNilo5VqqiC2VZIei4gDi85hneN/Nx9imFkF\nLggzS3JBdI9rig5gm2W7/3fzOQgzS/IehJkluSDMLMkFkTNJR0iaK2m+pPOKzmMdk3SdpOWS/lx0\nlqK5IHIkqR64CjgSGA+cIGl8samsCtcDRxQdoha4IPJ1EDA/Ip6PiPXATcCxBWeyDkTEPcCqonPU\nAhdEvoYCi8qmF2fzzLYKLoh8qZ15fl3ZthouiHwtBvYomx4GLC0oi1mnuSDy9Siwt6SRkhqB44Fb\nC85kVjUXRI4iogU4C5gJPAPMiIjZxaayjki6EXgQGCtpsaTTis5UFF9qbWZJ3oMwsyQXhJkluSDM\nLMkFYWZJLggzS3JBbGUktUp6UtKfJf1CUu8t2NYhkm7P7h9T6d2mkvpL+uRmjHGhpHOrnb/JOtdL\n+kgnxhrhd2B2LRfE1ufNiDggIvYF1gNnlC9USaf/XSPi1oi4tMIq/YFOF4Rt3VwQW7d7gdHZb85n\nJF0NzAL2kHS4pAclzcr2NPrAxs+nmCPpPuBDbRuSdIqkK7P7gyXdIump7DYZuBTYK9t7uTxb7/OS\nHpX0J0lfLtvW+dlnYPweGNvRk5B0eradpyT9apO9osMk3StpnqQp2fr1ki4vG/sTW/oXae1zQWyl\nJDVQ+pyJp7NZY4GfRMQ7gTXABcBhETEBeAw4R1JP4FrgaOC9wG6JzX8H+GNE7A9MAGYD5wHPZXsv\nn5d0OLA3pbe0HwBMlPQ+SRMpXVL+TkoFNKmKp/M/ETEpG+8ZoPzKxRHA+4GjgO9lz+E0YHVETMq2\nf7qkkVWMY53UUHQA67Rekp7M7t8L/BAYAiyMiIey+QdT+oCa+yUBNFK6dHgcsCAingWQdAMwvZ0x\n/h74J4CIaAVWSxqwyTqHZ7cnsuk+lAqjL3BLRKzNxqjmvSf7SvoKpcOYPpQuTW8zIyI2AM9Kej57\nDocD+5Wdn+iXjT2virGsE1wQW583I+KA8hlZCawpnwXcFREnbLLeAXTd280FfDUivr/JGJ/ZjDGu\nB6ZGxFOSTgEOKVu26bYiG/vsiCgvEiSN6OS41gEfYmybHgLeLWk0gKTeksYAc4CRkvbK1jsh8fg/\nAGdmj62XtBPwOqW9gzYzgVPLzm0MlbQrcA/wQUm9JPWldDjTkb7AMkk9gGmbLDtOUl2WeRQwNxv7\nzGx9JI2RtGMV41gneQ9iGxQRK7LfxDdK2iGbfUFEzJM0HbhD0svAfcC+7WziX4FrsncxtgJnRsSD\nku7PXkb8TXYe4h3Ag9kezBvAxyNilqSbgScpfRHzvVVE/nfg4Wz9p3lrEc0F/ggMBs6IiHWSfkDp\n3MQslQZfAUyt7m/HOsPv5jSzJB9imFmSC8LMklwQZpbkgjCzJBeEmSW5IMwsyQVhZkn/D/KFVxby\nvMoHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x20ccb044668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import itertools\n",
    "classes = [0,1]\n",
    "plt.figure()\n",
    "plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
    "plt.title('Confusion matrix')\n",
    "tick_marks = np.arange(len(classes))\n",
    "plt.xticks(tick_marks, classes)\n",
    "plt.yticks(tick_marks, classes, rotation=0)\n",
    "for i, j in  itertools.product(range(cm.shape[0]), range(cm.shape[1])):    \n",
    "    plt.text(j, i, cm[i, j],horizontalalignment=\"center\")\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>probability</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>0.996999</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0.995673</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>0.944646</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.942209</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.934389</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    probability  label\n",
       "92     0.996999      1\n",
       "81     0.995673      1\n",
       "71     0.944646      1\n",
       "80     0.942209      1\n",
       "98     0.934389      1"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sort_values('probability',inplace=True,ascending=False)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 50)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data[data['label']==0]),len(data[data['label']==1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "TPRandFPR=pd.DataFrame(index=range(len(data)),columns=('TPR','FPR'))\n",
    "for j in range(len(data)):\n",
    "    #以每一个概率为分类的阈值，统计此时正例和反例的数量，\n",
    "    data1=data.head(n=j+1)\n",
    "    \n",
    "    FPR=len(data1[data1['label']==0])/float(len(data[data['label']==0]))  #假正例的数量/反例的数量\n",
    "    TPR=len(data1[data1['label']==1])/float(len(data[data['label']==1])) #真正例的数量/正例的数量\n",
    "    \n",
    "    TPRandFPR.iloc[j]=[TPR,FPR]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TPR</th>\n",
       "      <th>FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    TPR FPR\n",
       "0  0.02   0\n",
       "1  0.04   0\n",
       "2  0.06   0\n",
       "3  0.08   0\n",
       "4   0.1   0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TPRandFPR.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VOX1+PHPYUkAQZStKpEJAloW\nMSKCVL9uCCIuWBdAo3VtWtS2qG1dqNaNH1Zbt6pVpBU1CCiK4gYqgguKgIjIIspOWBTCIluAMOf3\nx3MDkzBbktlz3q/XvJi565nLZM7c+9znPKKqGGOMMWVqJTsAY4wxqcUSgzHGmHIsMRhjjCnHEoMx\nxphyLDEYY4wpxxKDMcaYciwxmKiJSL6IvJ/sOFKJiGwTkaOSsN9cEVERqZPofceDiMwXkdOrsJ59\nJuPAEkOaEpHlIrLT+2JaJyIjRaRhPPepqqNUtXc89xFIRH4lIh+JyFYR2SIib4lIh0TtP0g8U0Xk\n+sBpqtpQVZfGaX9Hi8irIrLBe/9zReQWEakdj/1VlZeg2lZnG6raUVWnRtjPAckw0Z/JmsISQ3o7\nX1UbAnnA8cAdSY6nSoL96hWRHsD7wJvAEUBr4BtgWjx+oafaL28RaQN8CawCjlXVxsClQFegUYz3\nlbT3nmrH3XhU1R5p+ACWA2cFvH4IeCfgdTbwT2Al8CPwDFA/YH4/YA7wM7AE6ONNbwz8F1gLrAYe\nAGp7864GPvOePwP8s0JMbwK3eM+PAF4D1gPLgD8GLHcPMA4o9PZ/fZD39ynwdJDp7wEves9PB4qA\nO4EN3jHJj+YYBKx7G7AOeAk4FHjbi3mT9zzHW34osBcoAbYBT3rTFWjrPR8JPAW8A2zFfbG3CYin\nN7AI2AI8DXwc7L17yxYG/n8GmZ/r7fsq7/1tAIYEzO8GfAFs9v4vnwSyAuYrcCPwA7DMm/Y4LhH9\nDHwF/F/A8rW947zEe29fAUcCn3jb2u4dlwHe8ufhPl+bgc+BzhU+u7cBc4FdQB0CPs9e7LO8OH4E\nHvGmr/T2tc179CDgM+kt0xH4ANjorXtnsv9W0/GR9ADsUcX/uPJ/SDnAt8DjAfMfAyYATXC/MN8C\nhnnzunlfTr1wZ40tgV96894AngUOAloAM4DfefP2/RECp3pfIuK9PhTYiUsItbwvjruBLOAoYClw\ntrfsPcAe4EJv2foV3lsD3JfwGUHe9zXAWu/56UAp8AguCZzmfUEdE8UxKFv3H9669YGmwMXe/hsB\nrwJvBOx7KhW+yDkwMWz0jm8dYBQwxpvXzPuiu8ib9yfvGIRKDOuAa8L8/+d6+37Oi/043Jdse2/+\nCcBJ3r5ygYXA4Apxf+Adm7JkeYV3DOoAt3ox1PPm/QX3GTsGEG9/TSseA+91F+AnoDsuoVyF+7xm\nB3x25+ASS/2AaWWf5y+AK73nDYGTKrznOgH7upr9n8lGuCR4K1DPe9092X+r6fhIegD2qOJ/nPtD\n2ob79abAZOAQb57gviADf632YP8vw2eBR4Ns8xfel0vgmcVlwBTveeAfoeB+wZ3qvf4t8JH3vDuw\nssK27wCe957fA3wS5r3leO/pl0Hm9QH2eM9Px325HxQw/xXgriiOwenA7rIvvhBx5AGbAl5PJXJi\nGBEwry/wnff8N8AXAfMEl1hDJYY9eGdxIeaXfUnmBEybAQwMsfxgYHyFuM+M8BnbBBznPV8E9Aux\nXMXE8B/g/grLLAJOC/jsXhvk81yWGD4B7gWahXjPoRLDZcDX8fy7qykPu76X3i5U1Q9F5DTgZdyv\n0s1Ac9yv3q9EpGxZwf16A/dL7d0g2/MBdYG1AevVwn2BlaOqKiJjcH+MnwCX4y5/lG3nCBHZHLBK\nbdzloTIHbDPAJsAPHA58V2He4bjLJvuWVdXtAa9X4M5aIh0DgPWqWrJvpkgD4FFc8jnUm9xIRGqr\n6t4w8QZaF/B8B+4XL15M+96zd/yKwmynGPdeq7Q/ETkadybVFXcc6uDO4gKV+z8QkVuB671YFTgY\n95kC95lZEkU84P7/rxKRPwRMy/K2G3TfFVwH3Ad8JyLLgHtV9e0o9luZGE0Y1vicAVT1Y9yv1X96\nkzbgLut0VNVDvEdjdQ3V4P4o2wTZ1CrcGUOzgPUOVtWOIXY9GrhERHy4s4TXArazLGAbh6hqI1Xt\nGxh2mPezHXc54dIgs/vjzo7KHCoiBwW8bgWsieIYBIvhVtylku6qejDuchm4hBI25iisxZ0JuQ26\nbJUTenE+xF3Wqqr/4JJqO++93Mn+91Fm3/sRkf/DXffvDxyqqofgLjeWrRPqMxPMKmBohf//Bqo6\nOti+K1LVH1T1MtylzH8A47z/40jHvzIxmjAsMWSOx4BeIpKnqn7ctedHRaQFgIi0FJGzvWX/C1wj\nIj1FpJY375equhZ3J9C/RORgb14b74zkAKr6Na6hdgQwSVXLzhBmAD+LyG0iUl9EaotIJxE5sRLv\n53bcr84/ikgjETlURB7AXQ66t8Ky94pIlvfldh7wahTHIJhGuGSyWUSaAH+vMP9HXHtJVbwDHCsi\nF3p34twIHBZm+b8DvxKRh0XkMC/+tiJSKCKHRLG/Rrg2jW0i8ktgUBTLl+L+P+uIyN24M4YyI4D7\nRaSdOJ1FpKk3r+JxeQ74vYh095Y9SETOFZGo7qYSkStEpLn3f1j2mdrrxeYn9P/B28BhIjJYRLK9\nz033aPZpyrPEkCFUdT3wIu76Orhff4uB6SLyM+4X6DHesjNwjbiP4n4Vfow7/Qd3LTwLWIC7pDOO\n8Jc0RgNn4S5llcWyFzgfd41+Ge7X+wjcHU/Rvp/PgLNxjbVrcZeIjgdOUdUfAhZd58W5BtfY+3tV\nLbv8FPIYhPAYriF3AzAdmFhh/uO4M6RNIvJEtO/Fez8bcGdAD+EuE3XA3XmzK8TyS3BJMBeYLyJb\ncGdks3DtSpH8GXd5byvui3pshOUn4e74+h53rEsof7nnEVz7zfu4hPNf3LEC12b0gohsFpH+qjoL\n1+b0JO7/ZjGuLSBafXDveRvumA9U1RJV3YG7O2yat6+TAldS1a24GyrOx30ufgDOqMR+jafsjhJj\n0o7XU7ZQVcNdkklJIlILd7tsvqpOSXY8xgSyMwZjEkREzhaRQ0Qkm/3X/KcnOSxjDmCJwZjE6YG7\na2YD7nLHhaq6M7khGXMgu5RkjDGmnLidMYjI/0TkJxGZF2K+iMgTIrLYKw7WJV6xGGOMiV48O7iN\nxN2V8GKI+ecA7bxHd9x91xFvLWvWrJnm5ubGJkIT0caNG1mxYgV+v3/ftFq1auHzuZuYgs1r2rQp\nxcXFlVon1ttL5XVSIQaLO/XjjvV79fv9W70+LRHF9VKSiOQCb6tqpyDzngWmlnV6EZFFwOnevfQh\nde3aVWfNmhWHaE0wubm5rFix4oDp2dnZAOzaFfRuy6DCrRPr7aXyOqkQQ1XWSYUYqrJOKsSQqHUC\n5zXAdYUPsFtVs6PZRzJLYrSk/H3SRd60AxKDiBQABQCtWrVKSHDGWblyZdDplfkgR7NOrLeXyuuk\nQgxVWScVYqjKOqkQQ6LW2bVrF+2Bj3CdZSr8Is+Kdh/JvCupYvd8CNHlXVWHq2pXVe3avHnzOIdl\nAoVKxD6fb99pa0W1awcfRybcOrHeXiqvkwoxWNyJjSGR77Vxy5Y0wfXuq2B30JWCiWeFPlyvzXkh\n5j0LXBbwehFweKRtnnDCCWqqrrCwUH0+n4qI+nw+LSwsDDuvsLBQs7KyFJe0FdCsrKx98xo0aFBu\nXoMGDXTQoEFBp4dbJ9bbS+V1UiEGizv1467MOmeCfieiY0aM0MLCQm1Sr94B6wFLNdrv7mgXrMoj\nQmI4F9cFX3B142dEs01LDFVX1T+cunXrlptet27dfQklVKKpbAKKNC/T1kmFGCzu1I870jrtjjxS\n3wL1ew996aWQ6wGzNMrv7rg1PovIaFzN+2a4Ilt/x5V0RlWfERHB3bXUB9dGco26GithWeNz1cWy\nIdnn87F8+fJYhWaMqazXXoOrroLt2+Gww+Ctt6Br15CLi8hXqhp6gcBl45UY4sUSQ9XVqlWLWP1/\ni0i52+GMMQnk98PBB8OOHfCHP8Cjj0Kt8E3GlUkMVhKjBollQ7LdHWZMEjz/PCxb5pLAa6/BwoXw\n+OMRk0JlWWKoQfr27Rty+tChQ2nQoEG56Q0aNKCgoCDo9KFDh8YtTmNMBT/9BCecANdeC5d641ed\nfTYcE66KfNVZYqhB3n032Giebnp+fj7Dhw/H5/MhIvh8PoYPH87TTz8ddHp+fn6CozemhvrXv6Bl\nS5g9Gzp2hDfeiPsurY2hBgnVxmDtBcakqIEDYexYqFsXHn4Y/vSnKm/K2hgMo0aNIjc3l1q1apGb\nm8uoUaNCtgtYe4ExKWbjRvfvzTdDjx6wbl21kkJlWWLIQKNGjaKgoIAVK1agqqxYsYKCggLatm0b\ndPlQbQ/GmASbPx9atYJ27dydR927w+efQ5MmCQ3DEkMGGjJkCDt2lC+ftWPHDiZPnhx0+VBtD8aY\nBPH7YdAgOPZYWLUKjj8eSkuTFk4yi+iZOAlV+C5WyxtjYmjePDjzTFi/Hho2hFGj4IILkhqSnTFk\noFBtBtYnwZgUdMghsGUL/PrXUFyc9KQAlhjiIljDb7jpsV5n6NChZGWVr7CblZVlfRKMSRUffOD6\nIGzbBjk5LjG8/jpkRV0ZO76iLaqUKo9UL6KX7IqMZfNCFb4LV5TLGBNnO3eqnnOOq18qoprAvz9S\noYhevKR6P4ZQhepCSeRoVVb4zpgkeuUVuOYaV9/o8MPh3XchLy9hu7d+DElU2YbcXbt2VXoUp3Dr\nhJtnjczGJInf78pZ7NwJt9wCRUUJTQqVZYkhxirb8FuVAnbh1gk3zxqZjUmwESNgyRJX5G78eFi0\nyJW4iHHRu1hL7ejSUKjOYqeffnrIht+qFLALtU6kecaYBFi3zvVF+O1vYcAAN61XL9dxLR1E2xiR\nKo9Ub3z2+XzlGn3LHmUNvckercoYE2fDhqnWqeMamI89VnX16mRHpKrW+JxUVqjOmBqsf3949VVX\n9O6RR+Cmm5Id0T6VaXy2ns8x1qpVq6B3Jdn1fWMy2MaNrp7RX/7iLiNNmOA6rqUpa2OIsXCD4Rhj\nMsy8eXDkkdC2rbvz6MQT4ZNP0jopgCWGmAs3GI4xJkP4/VBQAJ07u1tPTzwxqUXvYs0uJcVYqL4C\n1ofAmAwxdy707AkbNkCjRjBmDGTYFQE7Y6iiUHWKbDAcYzJckybw889wySWubSHDkgJYYqiSUAPh\njBo1ytoYjMlE770HRx/tEkJODmzd6u4+qpOZF13sdtUqCFUPyeoUGZNhduyAiy6CSZNAxI2VcNll\nyY6qSux21TgL1V4QruaRtTEYk2ZGj4brrnP1jVq2dEXvOndOdlQJYZeSqiBUe4HVKTImQ/j9rpzF\nrl2ub0JRUY1JCmCJoUpCDYRjdYqMSXP/+c/+oncTJsDixfDQQ8mOKuEsMVRRxbaZstf5+fkMHz4c\nn8+HiODz+Rg+fDj5+fnJCNMYE401a9wZwQ037C96d+aZ0Lp1cuNKEmt8roJQjc/WwGxMGho6FP7+\nd9i7F447DiZOhMMOS3ZUMWeNz3FmndiMyRCXXgrjxrmxlp96Cn73u2RHlBIsMVSBFcozJs1t2ADN\nmsFtt8H69fDGG2lf3yiWrI2hCqwTmzFpas4cd+vp0Ue7O4+6doWpUy0pVGCJoQqsUJ4xaaZszOUu\nXVxDc/fuGVX0LtbsUlIVWBuDMWlk7lw44wxX1+jgg+GVV+Dss5MdVUqL6xmDiPQRkUUislhEbg8y\nv5WITBGRr0VkroikxbUYK5RnTBpp1gy2b4eBA6G42JJCFOKWGESkNvAUcA7QAbhMRDpUWOxvwCuq\nejwwEHg6XvHEkrUxGJPi3n4b2rRxRe+OOML9O3p0xha9i7V4njF0Axar6lJV3Q2MAfpVWEaBg73n\njYE1cYwnZqyNwZgUtWMH9OoF558Py5bBO++46RUqFZjw4pkYWgKrAl4XedMC3QNcISJFwLvAH4Jt\nSEQKRGSWiMxav359PGKtFGtjMCYFFRZC06bw4YduuM1vv03bSqjJFs/EIEGmVexmfRkwUlVzgL7A\nSyJyQEyqOlxVu6pq1+bNm8ch1NCCDchjbQzGpBi/H37/e9i9G26/HVauhI4dkx1V2opnYigCjgx4\nncOBl4quA14BUNUvgHpAszjGVCmhBuRp27Zt0OWtjcGYBHvySfjhB1f07u23YelSGDYs2VGlvbjV\nShKROsD3QE9gNTATuFxV5wcs8x4wVlVHikh7YDLQUsMElchaSaFqIoVitZKMSZCiIjjnHJg3D044\nAZJcPy0dVKZWUtzOGFS1FLgJmAQsxN19NF9E7hORC7zFbgV+KyLfAKOBq8MlhUSrbJuBtTEYkwD3\n3gu5uS4pHH+8G0DHxFRc791S1XdxjcqB0+4OeL4AODmeMVRHqJpItWvXZu/evUGXN8bE0cUXw+uv\nQ3Y2PPMMXH99siPKSFYSI4xQA/IUFBTYYDzGJIrfD+vWued33OF6Mf/0kyWFOLLEEEGwAXlOPvlk\nG4zHmESYPdsVvWvffn/Ru48+cqUtTNzYQD1h2IA8xiSJ3w/XXAMvvuhen3uuu4RkHdWqzAbqiRHr\nyGZMEsyZ44bV3LQJGjeG116Dnj2THVWNYpeSCN6JDaxYnjFJ0aIF7NwJ+fluQB1LCglX4xNDqE5s\no0aNsmJ5xiTKm2/CUUftL3q3dasrcWFF75KixrcxhGpHyM7OBmDXrl0HzLM2BmNiZNs2uOACmDIF\nRFwF1AEDkh1VRrI2hkoI1V4QLCFEWscYUwkvvODqG5WUQKtWMHGiu/vIJF2Nv5QUqr3A5/Ph8/kq\ntY4xJkp+P9x4oyt6d+edsGKFJYUUUuMTQ7h2hKFDh1pHNmNi6fHHyxe9W7YM7O8p5dT4xBBu0J38\n/HzryGZMLKxc6c4IBg/eP0bC6ae7S0gm5dT4xudatWod0LsZQETw+/0x248xNdbdd7uzgrKey++9\n58ZhNgmVEtVV04X1VTAmji66CO6/H+rWhf/9D2bOtKSQBmp8YrC+CsbEmN8Pa7wxuf72NzjrLNdR\n7ZprkhuXiVpUiUFEskQk+LBlaS5cG4MxppJmzoTDD3fDavr90KULfPABNGyY7MhMJURMDCJyLvAt\n8IH3Ok9Exsc7sESxekjGxEBpKVxxBXTr5kpin3aam2bSUjRnDPcB3YHNAKo6B8iYswdrYzCmmmbP\nhubNYdQoOOQQmDwZ3njDKqGmsWgSwx5V3VxhWnrdykToQnmhBuOxvgrGROmII1zv5d/8BoqLXWVU\nk9aiKYmxUET6A7VEpDXwJ2B6fMOKrbJCeTt27ADYVyivTLDBeIwxYbz2Gtx6qyuRfdhhruidFbzL\nGBH7MYjIQcDdQG9v0iTgXlXdGefYgqpKPwYrlGdMjPz8M5x/PnzyiSt6N2YM9O+f7KhMFGJdRO9s\nVb0NuC1gBxcBr1cxvoSzQnnGxMB//+vqG+3aBa1bu45qxxyT7KhMHETTxvC3INOGxDqQygjVXhBq\nnhXKM6aa/H5XzqK0FO65B5YutaSQwUKeMYjI2UAfoKWIPBIw62AgabUiNm7cGLa9INi8Hj16BL2U\n1LdvX04++eRy64AVyjNmn0cecZeO2rVzZwi5uZCTk+yoTJyFbGMQkeOBLrj2hfsCZm0FPlLVDfEP\n70DZ2dm6e/fuYNOB8JeHKiprRxg1ahRDhgxh5cqVtGrViqFDh1qhPFOzrVgBZ58NixbBCSdADOuT\nmeSoTBtDNI3P9VS1JCaRxYCIxOyWISuUZ0wQQ4bAgw+6y0fdurkzhSZNkh2VqaZYNz63FJGhQAeg\nXtlEVT26ivFVS1ZWFsHOGMraCoJdMqpduzZ79+49YLq1IxhTwYUXuvGX69WDZ591fRNMjRNN4/NI\n4HlAgHOAV4AxcYwprMaNGwedHm5gnYKCAhtwx5hQ/H4oKnLP777bXUIqLrakUJOpatgH8JX377cB\n0z6NtF68HllZWYrreV3u4fP5VFW1sLBQfT6fioj6fD4tLCwMO92YGu3zz1WbN1dt3Fh1z55kR2Pi\nCJilUX7PRtPG8DlwMjAemAisBv6pqkm5Vy1UG4O1FxhTCaWlcOWVroMauHETxo613ssZLNZtDDcD\nDYE/AkOBxsC1VQ+vekK1MVh7gTFRmjULevZ0vZibNIHx4+HUU5MdlUkhERODqn7pPd0KXAkgIkm7\nkblx48asX7/+gOk2sI4xUcrJgT174LrrYPhwqFXjx+syFYT9RIjIiSJyoYg08153FJEXSWIRvS1b\ntgSdbgPrGBPGK6+AzwebN7uidz//DCNGWFIwQYX8VIjIMGAUkA9MFJEhwBTgGyApt6oCQS8jgdU2\nMiaon3+G//s/GDAAVq2CDz90060twYQR7tPRDzhOVXeKSBNgjfd6UWJCC87aGIyJ0rPPwh//CLt3\nw1FHwcSJrrSFMRGEO48sUa+0tqpuBL6rbFIQkT4iskhEFovI7SGW6S8iC0Rkvoi8HGmbLVu2tIF1\njInE74c//xn27oX774clSywpmKiFO2M4SkTKSmsLkBvwGlW9KNyGRaQ28BTQCygCZorIBFVdELBM\nO+AO4GRV3SQiLaIJuuIttpFuuTWmxnjwQfj1r13l04kTXXnsI45IdlQmzYQrotcz3IqqOjnshkV6\nAPeo6tne6zu89YYFLPMQ8L2qjog24FBF9GxgHVOjLVsGvXvD4sXQtSvMnJnsiEyKiUk/hkhf/FFo\nCawKeF0EdK+wzNEAIjINqI1LJBMrbkhECoCCitMDWeOzqbFuuw3++U93+ahHD7A79Ew1xfPWBAky\nreLpSR2gHXA6kAN8KiKdVHVzuZVUhwPDIfQZgzU+mxqpXz+YMAHq13cjrF12WbIjMhkgnomhCDgy\n4HUO7s6mistMV9U9wDIRWYRLFCHPg62Dm6nxyoretWoF997rylu8+ipUKBRpTFVF3btFRLIrue2Z\nQDsRaS0iWcBAYEKFZd4AzvC23wx3aWlpuI1aBzdTo02bBi1aQOfOLiHk5cE771hSMDEVMTGISDcR\n+Rb4wXt9nIj8O9J6qloK3ARMAhYCr6jqfBG5T0Qu8BabBBSLyAJc57m/qGpxuO1aBzdTI5WWwqWX\nwimnuJLYvXolOyKTwaKprjodGAC8oarHe9PmqWqnBMR3ALsrydQ4M2e6ondbt0LTpm4gnZNPTnZU\nJs1U5q6kaC4l1VLVisOiHTgcWoKEG6jHmIx05JHujOH66+GnnywpmLiLJjGsEpFugIpIbREZDHwf\n57hCsjYGUyOMHu0SwsaNrujdtm3w3HNW9M4kRDSfskHALUAr4EfgJG9aUlgbg8lomze7vgiXXw5r\n1sBHH7nplhBMAkVzu2qpqg6MeyRRsiJ6JmM9/TQMHuzGSmjXDiZNciUtjEmwaH6GzBSRd0XkKhFp\nFPeIIrAieiYj+f2uB7MqDBsG339vScEkTcTEoKptgAeAE4BvReQNEUnqGYQV0TMZY9gwWLjQXSp6\n/303ZsLtQQsRG5MwEW9XLbewG5fhMSBfVWvHLaow7HZVkxGWLHFF75YutaJ3JiFieruqiDQUkXwR\neQuYAawHflXNGKvMGp9N2rv1VteGsHSpu/X0gw+SHZEx5UTT+DwPeAt4SFU/jXM8EVnjs0lr558P\nb7/tit49/7wbctOYFBNNYjhKVf1xjyRKVkTPpB2/37Ud+HxuNDVwRe/q1UtuXMaEEG6gnn+p6q0i\nMp4Dy2VHHMEtXqyNwaSVTz5xI6qVlroaR3XiWdDYmNBiMlAPMNb798nqhxQ71sZg0sLu3TBwIIwf\n717bOAkmjYQbwW2G97S9qpZLDiJyE1DdEd6qxNoYTMr78ks46yxXxqJ5c3jrLehecfBCY1JXNB3c\nrg0y7bpYBxItK6JnUl7r1q6j2qBBsG6dJQWTdkKeMYjIANzgOq1F5PWAWY2AzcHXij8romdSUmGh\n65g2d64bSOfnn62+kUlb4doYZgDFuCE5nwqYvhX4Op5BhWNtDCalbNwIffu6y0e1asGUKXDxxZYU\nTFoL18awDFgGfJi4cCKzNgaTMv79b9dZbc8eOOYYV/TO50t2VMZUW8ifNSLysffvJhHZGPDYJCIb\nExdiedbGYFKC3w933unaEh56CL77zpKCyRjh+jHUUlW/iAStiaSqSRnFzfoxmKS67z439nL79u7y\nUevWrk3BmBRXmX4M0Yz5nAusUdXdInIK0BkoVNWfqxtoVYhI0IBFBL8/ZTpom0yzaBH06QPLl8OJ\nJ8KMGRFXMSaVxHrM5zdww3q2AV4E2gMvVyO+aqk4FkMZa2MwceH3w5/+5M4Qli+H007bP6qaMRkq\nmsTgV9U9wEXAY6r6B6BlfMMKzQbqMQl1wQXwxBPQoAGMGwdTp0LDhsmOypi4impoTxG5FLgSuNCb\nVjd+IUVmA/WYuPL7YcUK137w//4f1K0LY8dCiLNVYzJNNG0MnYAbgM9VtVBEWgOXq2pSfqJb47OJ\nq6lT4aKLYO9eK3pnMkpMG5+9DdYB2novF6tqaTXiqxZrfDZxsXs39O8Pb77pXufnw8iRlhhMxohV\nddWyjf0f8BKwGhDgMBG5UlWnVS/MqrEObibmvvwSevaE7dvdradvv+3uPDKmhoqm8flRoK+qnqyq\nvwLOBR6Pb1ihWQc3E3OtW7t/b7oJ1q61pGBqvGgSQ5aqLih7oaoLgaS1wlkRPRMTL7wARxzhah2V\nFb3797+txpExRJcYZovIsyJyivf4D1ZEz6SrDRuga1e4+mr48Uf4+GM33RKCMftE89fwe2AJ8Ffg\nNmAp8Lt4BhWOdXAzVfbII3D44fDVV67D2rJlbthNY0w5YRufReRYoA0wXlUfSkxI4TVu3Jj169cf\nMN3aGExYfj/8/e/u+aOPwuDByY3HmBQWrrrqnbhyGPnAByISbCS3hLM2BlMpd98N8+e7S0UffeQa\nly0pGBNWuOqq84FuqrpdRJoD76pq0m/XsH4MJioLF8I557gezFb0zpiYFdHbparbAVR1fYRlQwXS\nR0QWichiEbk9zHKXiIiKSMTleH6wAAAbzklEQVSgrY3BhOX3w403QseOLimccYYVvTOmksK1MRwV\nMNazAG0Cx35W1YvCbdgbx+EpoBdQBMwUkQmBt756yzUC/gh8GU3A1sZgwjrvPHjvPTjoIBg1Cvr1\nS3ZExqSdcInh4gqvn6zktrvhymcsBRCRMUA/YEGF5e4HHgL+HM1GrY3BHKC01J0dtGkDDz4I9evD\n6NFW9M6YKgo35vPkam67JbAq4HUR0D1wARE5HjhSVd8WkZCJQUQKgIJwO7N+DDXUBx+4EdX8ftdZ\nrXNneO21ZEdlTFqLZ68eCTJtX8OxiNTCldu4NdKGVHW4qnZV1a7WxmAAKCmBc8+F3r1hyxZXEdU6\nqRkTE/H8SyoCjgx4nQOsCXjdCOgETBWR5cBJwIRIDdA2UI/hiy+gaVN491047DDXYW3kSEsMxsRI\n1H9JIpJdyW3PBNqJSGsRyQIGAhPKZqrqFlVtpqq5qpoLTAcuUNVZkTZsA/XUcO3auSQweDCsXg1d\nuiQ7ImMySsTEICLdRORb4Afv9XEi8u9I63ljNtwETAIWAq+o6nwRuU9ELqhqwKtXr2bPnj3lpu3Z\ns4chQ4ZUdZMmHYwY4cpZbNwIzZq5y0ePPmpnCcbEQTQjuE0HBgBvqOrx3rR5qtopAfEFi8c6uNUk\nP/3kOqrNng21a7uGZbsF1ZhKi1UHt33LqOqKCtP2Vj6s2LDG5xrk4YehZUuXFDp1guXLLSkYkwDR\nJIZVItINUBGpLSKDge/jHFdINlBPDeH3w/33gwg88QR8+y3k5CQ7KmNqhGgSwyDgFqAV8CPu7qFB\n8QwqHOvgluHuvLN80bt16+APf0h2VMbUKBHbGFKNtTFkqHnzXFtCURF06+bGYTbGxExl2hjCjsfg\nbew5AjqmlVHVsD2R4yUrKyvoKG7WxpCm/H644QYYPhxU4ayz4M03kx2VMTVaxMQAfBjwvB7wa8qX\nukgoK6KXYc49FyZOhIYNXX2j885LdkTG1HgRE4Oqjg18LSIvAR/ELaIIrI0hA5SWumE127Vzdx41\nagSFhVb0zpgUUek2BhFpA0xS1bbxCSni/q2NIZ1NmgT9+7tLSMXFlgyMSZBYtzFsYn8bQy1gIxBy\n0J14szaGNFVS4grdvfeeuwX12muhTjRXMo0xiRb2L1NEBDgOWO1N8muSb2OyNoY0NG0a9OoFO3e6\nshbvvgt5ecmOyhgTQth+DF4SGK+qe71H0u9ttTaGNHTMMe7s4M9/hjVrLCkYk+Ki6eA2Q0RSpnxl\nsMtIYAP1pJxnn4Vf/AI2bHBF7zZvdg3NxpiUFzIxiEjZZaZTcMlhkYjMFpGvRWR2YsI7kNVKSnHr\n1sFxx8Hvf+8alz//3E23KqjGpI1wbQwzgC7AhQmKJSotW7Zk9erV5c4cbKCeFDFsGNx1F+zd64bY\nfO89OOKIZEdljKmkcIlBAFR1SYJiiZoN1JOC/H6XGGrXhn//GwYlrZyWMaaaQvZjEJEi4JFQK6pq\nyHnxlJ2drcHaGXw+H8uXL098QDXdbbfBlVe6stizZ8NRR8EhhyQ7KmNMBbHqx1AbaIh35pAqrPE5\nRcydC337uqE1P/4Ypk+3ITaNyRDhEsNaVb0vYZFEyTq4JZnfD7/7Hfz3v67oXe/eMH58sqMyxsRQ\nuFtFUupMoYwN1JNkffu68ZcbNoR33nElLho0SHZUxpgYCnfG0DNhUVSCdXBLgtJSWLLEdVT717/g\n0EPhpZespIUxGcoG6jHhvf02XH65u2xkRe+MSVuVaXxOu15H1sEtQXbscO0H558P27bBZZfZGYIx\nNUTa/aVbEb0ECCx6l5PjOqp16pTsqIwxCZJ2ZwzWxpAA7du7S0Z//SusWmVJwZgaJu0Sg/VjiJOn\nn4YWLVzRuyZNYONG+Mc/kh2VMSYJ0i4xWBtDjK1Z4+oa3XgjbNrkOqqBFb0zpgZLu7/+li1bHpAc\nrIheFT3wALRqBd9+68ZIWLUKzjsv2VEZY5Is7RIDWBG9mPD73fgItWu7sRO+/hoOOyzZURljUkDa\n9WOwInrV4Pe7ondXXeUalOfMcUXvDj442ZEZY+KsMv0Y0i4xWAe3Kpozx5WzWLsWunff35ZgjKkR\nrIOb2c/vh2uugeOPd0mhb1+YOjXZURljUljaJQYroldJ55wDI0dC48YwcaIrfFevXrKjMsaksLRL\nDNbBLQq7d8PChe75Y49Bfr7rn3D22cmNyxiTFqyNIdNMmOASAVjRO2PMPinTxiAifURkkYgsFpHb\ng8y/RUQWiMhcEZksIr5I27Q2hhB27ICzzoJ+/WD7djfcphW9M8ZUQdwSg4jUBp4CzgE6AJeJSIcK\ni30NdFXVzsA44KFI27U2hiA++wyaNoXJk/d3WHv6aeu9bIypknh+c3QDFqvqUlXdDYwB+gUuoKpT\nVHWH93I6kBNpo9bGEESHDq5B+c47YcUK6Ngx2REZY9JYPBNDS2BVwOsib1oo1wHvBZshIgUiMktE\nZlkRPc/jj0OzZvDTT67o3aZNYGVBjDExEM/EEGzM6FANx1cAXYGHg81X1eGq2lVVu9b4NoaiIndG\nMHgwbNkCM2YkOyJjTIaJZ2IoAo4MeJ0DrKm4kIicBQwBLlDVXZE2WqPbGO65B3JzYcECOOEEWL3a\nit4ZY2Iubrerikgd4HugJ7AamAlcrqrzA5Y5Htfo3EdVf4hmuzW2VpLfD4ceCrt2wVNPwXXXJTsi\nY0waSYnbVVW1FLgJmAQsBF5R1fkicp+IXOAt9jDQEHhVROaIyIRI261RbQx+P9x8M8yd6+4w+vRT\n11HNkoIxJo7SroNbjTljmDULzj8f1q2DHj3g88+THZExJo2lxBlDvGT8QD1+P/zmN3DiiS4pnHee\nFb0zxiRUWnaNzeiBevr0gQ8+gEMOgXHjoGfPZEdkTJXt2bOHoqIiSkpKkh1KjVGvXj1ycnKoW7du\nlbdhl5JSwe7d8MMP7jbUhQvhH/+AESOspIVJe8uWLaNRo0Y0bdoUkWB3sJtYUlWKi4vZunUrrVu3\nLjcvoy8lZVzj8/jxroNa9+4uQbRv78pkW1IwGaCkpMSSQgKJCE2bNq32GVraJYaM6eC2bRuccQZc\ndJErgHfttZYMTEaypJBYsTjeafdN1LhxY9avX3/A9LTq4PbJJ9C7t+uT4PPBpElwzDHJjsoYY4A0\nPGPIiCJ6nTpBgwZw112wfLklBWPibPz48YgI33333b5pU6dO5bwKlQOuvvpqxo0bB7iG89tvv512\n7drRqVMnunXrxnvvBS3nVinDhg2jbdu2HHPMMUyaNCnoMh999BFdunShU6dOXHXVVZSWlpaLOy8v\nj44dO3LaaadVO55g0i4xpG0bwyOPuNLYZUXvNm6E++5LdlTG1AijR4/mlFNOYcyYMVGvc9ddd7F2\n7VrmzZvHvHnzeOutt9i6dWu14liwYAFjxoxh/vz5TJw4kRtuuIG9e/eWW8bv93PVVVcxZswY5s2b\nh8/n44UXXgBg8+bN3HDDDUyYMIH58+fz6quvViueUNLuUlJWVlbQ5JCybQwrV7ohNb/7zrUhzJoF\n6XTZy5gYGTx4MHPmzInpNvPy8njsscfCLrNt2zamTZvGlClTuOCCC7jnnnsibnfHjh0899xzLFu2\njOzsbAB+8Ytf0L9//2rF++abbzJw4ECys7Np3bo1bdu2ZcaMGfTo0WPfMsXFxWRnZ3P00UcD0KtX\nL4YNG8Z1113Hyy+/zEUXXbTv+65FixbViieUtDtjSKsienfdBa1bu6Rw4omwdq0lBWMS7I033qBP\nnz4cffTRNGnShNmzZ0dcZ/HixbRq1YqDDz444rI333wzeXl5BzwefPDBA5ZdvXo1Rx65v7ZoTk4O\nq1evLrdMs2bN2LNnD7NmzQJg3LhxrFrlRjD4/vvv2bRpE6effjonnHACL774YsT4qiLtzhjSpo3B\n74cnnnBjLj/7rOvNbEwNFumXfbyMHj2awYMHAzBw4EBGjx5Nly5dQt69U9m7eh599NGolw3Wb6zi\n/kSEMWPGcPPNN7Nr1y569+5NHe+OxdLSUr766ismT57Mzp076dGjByeddNK+s4tYSbvEkNJtDH6/\nGyfh2mshLw+mTXNlshs2THZkxtRIxcXFfPTRR8ybNw8RYe/evYgIDz30EE2bNmXTpk3llt+4cSPN\nmjWjbdu2rFy5kq1bt9KoUaOw+7j55puZMmXKAdMHDhzI7beXH+o+Jydn369/gKKiIo444ogD1u3R\noweffvopAO+//z7ff//9vvWbNWvGQQcdxEEHHcSpp57KN998E/PEgKqm1SMrK0txA/6Ue/h8Pk2q\nGTNUW7RQBdUePZIbizEpYsGCBUnd/zPPPKMFBQXlpp166qn6ySefaElJiebm5u6Lcfny5dqqVSvd\nvHmzqqr+5S9/0auvvlp37dqlqqpr1qzRl156qVrxzJs3Tzt37qwlJSW6dOlSbd26tZaWlh6w3I8/\n/qiqqiUlJXrmmWfq5MmTVdUdzzPPPFP37Nmj27dv144dO+q33357wPrBjjswS6P8nrU2huoqLYXL\nL4du3dwdRxdeaEXvjEkRo0eP5te//nW5aRdffDEvv/wy2dnZFBYWcs0115CXl8cll1zCiBEj9n3H\nPPDAAzRv3pwOHTrQqVMnLrzwQpo3b16teDp27Ej//v3p0KEDffr04amnnqJ27dqA+w5bs8aNZfbw\nww/Tvn17OnfuzPnnn8+ZZ54JQPv27enTpw+dO3emW7duXH/99XTq1KlaMQVjtZKqq1cv+PBDN4jO\n66/D6acnPgZjUtTChQtp3759ssOocYIdd6uVFG8lJTBvnnv+xBNwzTVuAB1LCsaYDJB2iSHptZLG\njXMd1U46aX/Ru//9z42wZowxGSDtvs2SNlDPzz/DqafCpZfCzp1QUGBF74wxGSktv9kqtovEvZ1k\n6lQ3gM6uXXDUUTBxIrRrF999GmNMkqTdGcPq1avZs2dPuWl79uxhyJAh8dtp585w0EFw772wZIkl\nBWNMRku7M4aENT4//DAMG+bKWbRoAcXFsd2+McakqLQ7Y4h74/OKFa4M9l//6gbT+eqr2GzXGJM0\nO3fu5LTTTmPJkiXUr1+/XE2j3bt3M3LkSJo3b05eXh4dOnTgueeeAyg3/Ze//GXQ8hfPP//8vm1l\nZWVx7LHHkpeXx+23316l7T755JM8//zziTkwoUTbEy5VHs2bNw/a83nQoEFBexpWyh13qNaq5Xov\nd++uWlxc/W0aU4NVtudzYWGh+nw+FRH1+XxaWFgYkziefPJJfeyxx3TZsmXasWPHA+Y///zzeuON\nN6qq63XcrFkzXbduXbnpGzZs0KZNm+rKlStD7sfn8+n69eurtd3t27drXl5etd5vjev5HLcien4/\nPP20K3r30kswfbobN8EYkxCjRo2ioKCAFStWoKqsWLGCgoICRo0aFZNt9+vXL6plW7RoQZs2bVix\nYkW56U2bNqVt27asXbu2SjFEu90GDRqQm5vLjBkzqrSfWEi7xBDTNga/HwYNgtmzXT+EadNcW8IV\nV1QzSmNMZQ0ZMoQdO3aUm7Zjx45q31iye/duli5dSm5uLgBLlizZd+nnxhtvPGD5pUuXsnTpUtq2\nbVtu+sqVKykpKaFz585ViqMy2+3ateu+InrJkHaNzzEbqOeLL6BfP1i/HubOdUmhY8cYRWmMqaxQ\nP+6qe2PJhg0bOOSQQ/a9btOmTdABg8aOHctnn31GdnY2zz77LE28KwZjx45lypQpLFq0iOeee456\n9epVav9V2W6LFi3KDUOaaGl3xlDtInqlpTBwIPzqVy4pXHwxfPxxDCM0xlRFqB931b2xpH79+pSU\nlERcbsCAAcyZM4cvv/yyXOG9AQMGMH/+fD799FNuvfVW1q1bV6n9V2W7JSUl1K9fv1L7iaW0SwzV\nbmM45xwYO9a1H3z6qStxYT2YjUm6oUOH0qBBg3LTGjRoUO2qBoceeih79+6NKjmE06NHD6688koe\nf/xxAMaPH88dd9xRrW0G2y64kdriUTU1WmmXGKrUxlBS4i4XATz5JPz2t+5s4ZRT4hChMaYq8vPz\nGT58OD6fDxHB5/MxfPhw8vPzq73t3r1789lnn1V7O7fddhvPP/88W7duZcmSJVEN/VnZ7QJMmzaN\ns846KybbrpJob19KlUelB+oZM0a1fn3Vgw5S9QbcMMYkRrIH6ikze/ZsveKKK2K6zfz8fP3pp59i\nuk3V2MRa3dtV0+4aSuPGjVm/fv0B0w9oY9i8Gc47zzUq16oFt9zibkU1xtQ4xx9/PGeccQZ79+7d\nNzBOdRUWFsZkOxVt2LCB+++/Py7bjlbaJYao2himToWzz3Zlsdu0gUmT3L/GmBrr2muvTXYIUenV\nq1eyQ8jQNoa8PGjUCB54ABYvtqRgTBJpmo0Sme5icbzT7owhVD+GYQcf7IbXXLgQDjvMjahmjEmq\nevXqUVxcTNOmTRGRZIeT8VSV4uLiSve1qCiuiUFE+gCPA7WBEar6YIX52cCLwAlAMTBAVZeH22bL\nli358ccf9/WQbA28L0LbLVtcG8I337jEYIxJupycHIqKioK2C5r4qFevHjk5OdXaRtwSg4jUBp4C\negFFwEwRmaCqCwIWuw7YpKptRWQg8A9gQLjtNmnShPvvv58hQ4Zw44oV3AqIquuw9s47ENDD0RiT\nXHXr1qV169bJDsNUUjzbGLoBi1V1qaruBsYAFatY9QNe8J6PA3pKFOeb+fn5LF+6lL80bkyt+vWR\nMWPc3UeWFIwxptrieSmpJbAq4HUR0D3UMqpaKiJbgKZAuQYCESkACiCge3ytWq7ekc8HFXpLGmOM\nqbp4njEE++Vfsbk8mmVQ1eGq2lVVuzZv3nz/jPbtLSkYY0yMxfOMoQg4MuB1DrAmxDJFIlIHaAxs\nDLfRr776aoOIlBU0b0aFs4sayo6DY8fBjkEZOw5O4HHwRbtSPBPDTKCdiLQGVgMDgcsrLDMBuAr4\nArgE+Egj3ISrqvtOGURklqp2jWnUaciOg2PHwY5BGTsOTlWPQ9wSg9dmcBMwCXe76v9Udb6I3Ier\n2TEB+C/wkogsxp0pDIxXPMYYY6IT134Mqvou8G6FaXcHPC8BLo1nDMYYYyon7UpiVDA82QGkCDsO\njh0HOwZl7Dg4VToOYnVMjDHGBEr3MwZjjDExZonBGGNMOWmRGESkj4gsEpHFInJ7kPnZIjLWm/+l\niOQmPsr4i+I43CIiC0RkrohMFpGo71tOF5GOQcByl4iIikhG3rIYzXEQkf7e52G+iLyc6BgTIYq/\niVYiMkVEvvb+LvoG2046E5H/ichPIjIvxHwRkSe8YzRXRLpE3Gi0Q70l64G71XUJcBSQBXwDdKiw\nzA3AM97zgcDYZMedpONwBtDAez4o045DNMfAW64R8AkwHeia7LiT9FloB3wNHOq9bpHsuJN0HIYD\ng7znHYDlyY47DsfhVKALMC/E/L7Ae7hKEycBX0baZjqcMcStGF+aiXgcVHWKqu7wXk7H9TbPJNF8\nFgDuBx4CShIZXAJFcxx+CzylqpsAVPWnBMeYCNEcBwUO9p435sDqC2lPVT8hfMWIfsCL6kwHDhGR\nw8NtMx0SQ7BifC1DLaOqpUBZMb5MEs1xCHQd7ldCJol4DETkeOBIVX07kYElWDSfhaOBo0VkmohM\n98ZGyTTRHId7gCtEpAjXp+oPiQktpVT2uyMtRnCLWTG+NBf1exSRK4CuwGlxjSjxwh4DEakFPApc\nnaiAkiSaz0Id3OWk03Fnjp+KSCdV3Rzn2BIpmuNwGTBSVf8lIj1wlRY6qao//uGljEp/P6bDGUNl\nivERbTG+NBTNcUBEzgKGABeo6q4ExZYokY5BI6ATMFVEluOup07IwAboaP8m3lTVPaq6DFiESxSZ\nJJrjcB3wCoCqfgHUwxWWq0mi+u4IlA6JYV8xPhHJwjUuT6iwTFkxPoiyGF8aingcvMsoz+KSQiZe\nUw57DFR1i6o2U9VcVc3FtbNcoKqzkhNu3ETzN/EG7mYERKQZ7tLS0oRGGX/RHIeVQE8AEWmPSww1\nbZzRCcBvvLuTTgK2qOracCuk/KUktWJ8QNTH4WGgIfCq1/a+UlUvSFrQMRblMch4UR6HSUBvEVkA\n7AX+oqrFyYs69qI8DrcCz4nIzbjLJ1dn2o9GERmNu2TYzGtL+TtQF0BVn8G1rfQFFgM7gGsibjPD\njpExxphqSodLScYYYxLIEoMxxphyLDEYY4wpxxKDMcaYciwxGGOMKccSg0k5IrJXROYEPHLDLJsb\nqqpkJfc51avS+Y1XRuKYKmzj9yLyG+/51SJyRMC8ESLSIcZxzhSRvCjWGSwiDaq7b1NzWGIwqWin\nquYFPJYnaL/5qnocriDjw5VdWVWfUdUXvZdXA0cEzLteVRfEJMr9cT5NdHEOBiwxmKhZYjBpwTsz\n+FREZnuPXwVZpqOIzPDOMuaKSDtv+hUB058VkdoRdvcJ0NZbt6dXy/9br+59tjf9Qdk/9sU/vWn3\niMifReQSXK2qUd4+63u/9LuKyCAReSgg5qtF5N9VjPMLAoqhich/RGSWuPEX7vWm/RGXoKaIyBRv\nWm8R+cI7jq+KSMMI+zE1jCUGk4rqB1xGGu9N+wnopapdgAHAE0HW+z3wuKrm4b6Yi7wyCAOAk73p\ne4H8CPs/H/hWROoBI4EBqnosrlLAIBFpAvwa6KiqnYEHAldW1XHALNwv+zxV3RkwexxwUcDrAcDY\nKsbZB1f6oswQVe0KdAZOE5HOqvoEri7OGap6hlce42/AWd6xnAXcEmE/poZJ+ZIYpkba6X05BqoL\nPOldU9+Lq/1T0RfAEBHJAV5X1R9EpCdwAjDTKxNSH5dkghklIjuB5bjyzMcAy1T1e2/+C8CNwJO4\nsR5GiMg7QNQlvlV1vYgs9WrW/ODtY5q33crEeRCuDETgaFz9RaQA93d9OG5gmrkV1j3Jmz7N208W\n7rgZs48lBpMubgZ+BI7DnekeMAiPqr4sIl8C5wKTROR6XMnhF1T1jij2kR9YcE9Ego7p4dXo6YYr\nzjYQuAk4sxLvZSzQH/gOGK+qKu5bOuo4caOVPQg8BVwkIq2BPwMnquomERmJKxhXkQAfqOpllYjX\n1DB2Kcmki8bAWq+O/pW4X8vliMhRwFLv8skE3CWVycAlItLCW6aJRD8W9ndAroi09V5fCXzsXZNv\nrKrv4hp2g90ZtBVXBjyY14ELcWMFjPWmVSpOVd2DuyR0kncZ6mBgO7BFRH4BnBMilunAyWXvSUQa\niEiwsy9Tg1liMOniaeAqEZmOu4y0PcgyA4B5IjIH+CVuOMMFuC/Q90VkLvAB7jJLRKpagqtE+aqI\nfAv4gWdwX7Jve9v7GHc2U9FI4JmyxucK290ELAB8qjrDm1bpOL22i38Bf1bVb3BjPM8H/oe7PFVm\nOPCeiExR1fW4O6ZGe/uZjjtWxuxj1VWNMcaUY2cMxhhjyrHEYIwxphxLDMYYY8qxxGCMMaYcSwzG\nGGPKscRgjDGmHEsMxhhjyvn/MLbecYi/ql4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x20ccb0f1748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import auc\n",
    "AUC= auc(TPRandFPR['FPR'],TPRandFPR['TPR'])\n",
    "plt.scatter(x=TPRandFPR['FPR'],y=TPRandFPR['TPR'],label='(FPR,TPR)',color='k')\n",
    "plt.plot(TPRandFPR['FPR'], TPRandFPR['TPR'], 'k',label='AUC = %0.2f'% AUC)\n",
    "plt.legend(loc='lower right')\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot([(0,0),(1,1)],'r--')\n",
    "plt.xlim([-0.01,1.01])\n",
    "plt.ylim([-0.01,01.01])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
